# -*- coding: utf-8 -*-
import scrapy
from scrapy.http import Request
from urllib import parse
import re
import datetime

from ..items import JobBoleItem, date_convert
from ..utils.common import get_md5


class JobboleSpider(scrapy.Spider):
    name = "jobbole"
    allowed_domains = ["blog.jobbole.com"]
    start_urls = ['http://blog.jobbole.com/all-posts/']

    def parse(self, response):
        post_nodes = response.xpath('//div[@id="archive"]//div[@class="post-thumb"]/a')
        for post_node in post_nodes:
            thumbnail = post_node.xpath('img/@src').extract_first('')
            post_url = post_node.xpath('@href').extract_first('')
            yield Request(url=parse.urljoin(response.url, post_url), meta={'thumbnail': thumbnail}, callback=self.parse_detail)

        next_url = response.xpath('//a[@class="next page-numbers"]/@href').extract_first('')
        if next_url:
            yield Request(url=parse.urljoin(response.url, next_url), callback=self.parse)

    def parse_detail(self, response):
        # 缩略图
        thumbnail = response.meta.get('thumbnail', '')
        # 标题
        title = response.xpath(
            '//div[@class="entry-header"]/h1/text()').extract_first('')

        #创建时间
        create_time = response.xpath(
            '//p[@class="entry-meta-hide-on-mobile"]/text()'
        ).extract_first('').replace('·', '').strip()

        # 点赞数
        praise_nums = response.xpath('//div[@class="post-adds"]/span[1]/h10/text()').extract_first('0')

        # 收藏数
        fav_nums = response.xpath('//span[contains(@class, "bookmark-btn")]/text()').extract_first('0')
        match_re = re.match('.*?(\d+)', fav_nums)
        if match_re:
            fav_nums = match_re.group(1)
        else:
            fav_nums = 0

        # 评论数
        comment_nums = response.xpath('//a[@href="#article-comment"]/span/text()').extract_first('0')
        match_re = re.match('.*?(\d+)', comment_nums)
        if match_re:
            comment_nums = match_re.group(1)
        else:
            comment_nums = 0

        # 正文内容
        content = response.xpath('//div[@class="entry"]').extract_first('')

        # 文章标签
        tag_list = response.xpath(
            '//p[@class="entry-meta-hide-on-mobile"]/a[contains(@href, "tag")]/text()'
        ).extract()
        tags = ','.join(tag_list)

        # 日期
        # try:
        #     create_time = datetime.datetime.strptime(create_time, '%Y/%m/%d').date()
        # except Exception as e:
        #     create_time = datetime.datetime.now().date()

        # 保存字段
        jobbole_item = JobBoleItem()
        jobbole_item['title'] = title
        jobbole_item['url'] = response.url
        jobbole_item['url_object_id'] = get_md5(response.url)
        jobbole_item['thumbnail'] = [thumbnail]
        jobbole_item['create_time'] = date_convert(create_time)
        jobbole_item['praise_nums'] = praise_nums
        jobbole_item['fav_nums'] = fav_nums
        jobbole_item['comment_nums'] = comment_nums
        jobbole_item['content'] = content
        jobbole_item['tags'] = tags
        yield jobbole_item

